{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 下载需要的包"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "pip install transformers\n",
    "pip install modelscope\n",
    "pip install datasets\n",
    "pip install accelerate\n",
    "pip install bitsandbytes \n",
    "pip install peft\n",
    "pip install swanlab\n",
    "pip install sentencepiece\n",
    "pip install trl \n",
    "pip install deepspeed\n",
    "\n",
    "\n",
    "pip install unsloth"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 下载数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess\n",
    "\n",
    "import os\n",
    "os.environ[\"HF_ENDPOINT\"] = \"https://hf-mirror.com\"\n",
    "\n",
    "from datasets import load_dataset\n",
    "\n",
    "ds = load_dataset(\"BAAI/IndustryInstruction_Health-Medicine\",cache_dir = './data/Medicine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds['test'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from unsloth.chat_templates import standardize_sharegpt\n",
    "dataset = standardize_sharegpt(ds['test'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modelscope import AutoModelForCausalLM, AutoTokenizer\n",
    "import torch\n",
    "import os\n",
    "from datasets import load_dataset,Dataset\n",
    "from swanlab.integration.transformers import SwanLabCallback\n",
    "import pandas as pd\n",
    "from trl import SFTTrainer, SFTConfig\n",
    "from transformers import TextStreamer\n",
    "from peft import LoraConfig, TaskType, get_peft_model\n",
    "import deepspeed\n",
    "\n",
    "DS_CONFIG = \"ds_z2_offload_config.json\"\n",
    "\n",
    "from typing import Optional, List, Union\n",
    "import sys\n",
    "\n",
    "\n",
    "model_name = \"/root/autodl-tmp/Qwen/Qwen3-8B\"\n",
    "\n",
    "# load the tokenizer and the model\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "non_reasoning_conversations = tokenizer.apply_chat_template(\n",
    "    dataset[\"conversations\"],\n",
    "    tokenize = False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "non_reasoning_conversations[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多卡训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "deepspeed --include 'localhost:0,1' train.py"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "qwen2.5vl",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
